package a_aa_amainpackage.data_deal.tm_deal

import com.alibaba.fastjson.{JSON, JSONObject}
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkConf, SparkContext}
import org.elasticsearch.spark._
object Tm_newcateadd5 {



  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("SparktoES")
    conf.set("es.nodes", "192.168.1.157")
    conf.set("es.port", "9200")
    conf.set("cluster.name", "O2OElastic")
    conf.set("es.index.read.missing.as.empty", "true")
    conf.setMaster("local[*]")

    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
    sc.setLogLevel("WARN")
    sc.hadoopConfiguration.set("fs.s3a.access.key", "GAO7EO9FWKPJ8WFCQDME")
    sc.hadoopConfiguration.set("fs.s3a.secret.key", "LZ0xaHBSYKHaJ9ECDbX9f7zin79UZkXfGoNapRPL")
    sc.hadoopConfiguration.set("fs.s3a.endpoint", "https://obs.cn-north-1.myhuaweicloud.com")
    val platform = "tmall"
    val month = "1"

    /**
      *
      */
    val values = sc.esJsonRDD(s"2020_${platform}/${platform}_2020_${month}",
      """
        |{
        |  "size": 0,
        |  "query": {
        |    "term": {
        |      "firstCategoryId": {
        |        "value": "10099"
        |      }
        |    }
        |  },"aggs": {
        |    "NAME": {
        |      "terms": {
        |        "field": "categoryId",
        |        "size": 500
        |      },"aggs": {
        |        "NAME": {
        |          "terms": {
        |            "field": "title.keyword",
        |            "size": 10
        |          }
        |        }
        |      }
        |    }
        |  }
        |}
      """.stripMargin).values

    val data = sqlContext.read.json(values)
    data.registerTempTable("data")
    //原始数据保存一份
    //data.write.json("s3a://o2o-dataproces-group/panzonghao/TM/2019/12/tmall_new_cate")
    val datas = sqlContext.sql(
      """
         |select
         |*,
         |case --2509
         |when categoryId  =  '201211201' then  '10025'
         |--when categoryId  =  '201247207' then  '10011'
         |--when categoryId  =  '201215402' then  '10017'
         |--when categoryId  =  '50023727'  then  '10099'
         |--when categoryId  =  '201241307' then  '10011'
         |--when categoryId  =  '201249513' then  '10022'
         |--when categoryId  =  '201249208' then  '10022'
         |--when categoryId  =  '200800004' then  '10017'
         |--when categoryId  =  '201270781' then  '10018'
         |--when categoryId  =  '50023729'  then  '10099'
         |--when categoryId  =  '201254109' then  '10015'
         |--when categoryId  =  '201217430' then  '10026'
         |--when categoryId  =  '201219703' then  '10022'
         |--when categoryId  =  '201248402' then  '10011'
         |--when categoryId  =  '201249804' then  '10022'
         |--when categoryId  =  '201267573' then  '10022'
         |--when categoryId  =  '201219811' then  '10022'
         |--when categoryId  =  '201234104' then  '10022'
         |--when categoryId  =  '201173712' then  '10013'
         |--when categoryId  =  '201183501' then  '10022'
         |--when categoryId  =  '201169116' then  '10016'
         |--when categoryId  =  '201220510' then  '10022'
         |--when categoryId  =  '201248002' then  '10013'
         |--when categoryId  =  '201250101' then  '10013'
         |--when categoryId  =  '201229810' then  '10022'
         |--when categoryId  =  '201230514' then  '10011'
         |--when categoryId  =  '201236601' then  '10022'
         |--when categoryId  =  '201237616' then  '10022'
         |--when categoryId  =  '201228511' then  '10029'
         |--when categoryId  =  '201221102' then  '10022'
         |--when categoryId  =  '200926002' then  '10011'
         |--when categoryId  =  '201249901' then  '10013'
         |
         |
         |
         |else firstCategoryId end as firstCate,
         |
         |case
         |when categoryId  =  '201211201' then  '1002507'
         |--when categoryId  =  '201247207' then  '1001102'
         |--when categoryId  =  '201215402' then  '1001704'
         |--when categoryId  =  '50023727'  then  '1009999'
         |--when categoryId  =  '201241307' then  '1001102'
         |--when categoryId  =  '201249513' then  '1002205'
         |--when categoryId  =  '201249208' then  '1002206'
         |--when categoryId  =  '200800004' then  '1001706'
         |--when categoryId  =  '201270781' then  '1001804'
         |--when categoryId  =  '50023729'  then  '1009999'
         |--when categoryId  =  '201254109' then  '1001501'
         |--when categoryId  =  '201217430' then  '1002604'
         |--when categoryId  =  '201219703' then  '1002209'
         |--when categoryId  =  '201248402' then  '1001101'
         |--when categoryId  =  '201249804' then  '1002206'
         |--when categoryId  =  '201267573' then  '1002206'
         |--when categoryId  =  '201219811' then  '1002206'
         |--when categoryId  =  '201234104' then  '1002401'
         |--when categoryId  =  '201173712' then  '1001399'
         |--when categoryId  =  '201183501' then  '1002206'
         |--when categoryId  =  '201169116' then  '1001605'
         |--when categoryId  =  '201220510' then  '1002206'
         |--when categoryId  =  '201248002' then  '1001302'
         |--when categoryId  =  '201250101' then  '1001305'
         |--when categoryId  =  '201229810' then  '1002206'
         |--when categoryId  =  '201230514' then  '1001105'
         |--when categoryId  =  '201236601' then  '1002205'
         |--when categoryId  =  '201237616' then  '1002206'
         |--when categoryId  =  '201228511' then  '1002999'
         |--when categoryId  =  '201221102' then  '1002209'
         |--when categoryId  =  '200926002' then  '1001199'
         |--when categoryId  =  '201249901' then  '1001304'
         |
         |else secondCategoryId end as secondCate,
         |case
         |
         |when categoryId  =  '201211201' then  '100250799'
         |--when categoryId  =  '201247207' then  '100110299'
         |--when categoryId  =  '201215402' then  '100170401'
         |--when categoryId  =  '50023727'  then  '100999999'
         |--when categoryId  =  '201241307' then  '100110299'
         |--when categoryId  =  '201249513' then  '100220599'
         |--when categoryId  =  '201249208' then  '100220699'
         |--when categoryId  =  '200800004' then  '100170699'
         |--when categoryId  =  '201270781' then  '100180499'
         |--when categoryId  =  '50023729'  then  '100999999'
         |--when categoryId  =  '201254109' then  '100150199'
         |--when categoryId  =  '201217430' then  '100260499'
         |--when categoryId  =  '201219703' then  '100220999'
         |--when categoryId  =  '201248402' then  '100110199'
         |--when categoryId  =  '201249804' then  '100220699'
         |--when categoryId  =  '201267573' then  '100220699'
         |--when categoryId  =  '201219811' then  '100220699'
         |--when categoryId  =  '201234104' then  '100240106'
         |--when categoryId  =  '201173712' then  '100139999'
         |--when categoryId  =  '201183501' then  '100220699'
         |--when categoryId  =  '201169116' then  '100160599'
         |--when categoryId  =  '201220510' then  '100220699'
         |--when categoryId  =  '201248002' then  '100130299'
         |--when categoryId  =  '201250101' then  '100130599'
         |--when categoryId  =  '201229810' then  '100220699'
         |--when categoryId  =  '201230514' then  '100110504'
         |--when categoryId  =  '201236601' then  '100220502'
         |--when categoryId  =  '201237616' then  '100220699'
         |--when categoryId  =  '201228511' then  '100299999'
         |--when categoryId  =  '201221102' then  '100220999'
         |--when categoryId  =  '200926002' then  '100119999'
         |--when categoryId  =  '201249901' then  '100130499'
         |
         |
         |else thirdCategoryId end as thirdCate,
         |
         |
         |case
         |
         |when categoryId  =  '201211201' then  '10025079999'
         |--when categoryId  =  '201247207' then  '10011029999'
         |--when categoryId  =  '201215402' then  '10017040199'
         |--when categoryId  =  '50023727'  then  '10099999999'
         |--when categoryId  =  '201241307' then  '10011029999'
         |--when categoryId  =  '201249513' then  '10022059999'
         |--when categoryId  =  '201249208' then  '10022069999'
         |--when categoryId  =  '200800004' then  '10017069999'
         |--when categoryId  =  '201270781' then  '10018049999'
         |--when categoryId  =  '50023729'  then  '10099999999'
         |--when categoryId  =  '201254109' then  '10015019999'
         |--when categoryId  =  '201217430' then  '10026049999'
         |--when categoryId  =  '201219703' then  '10022099999'
         |--when categoryId  =  '201248402' then  '10011019999'
         |--when categoryId  =  '201249804' then  '10022069999'
         |--when categoryId  =  '201267573' then  '10022069999'
         |--when categoryId  =  '201219811' then  '10022069999'
         |--when categoryId  =  '201234104' then  '10024010699'
         |--when categoryId  =  '201173712' then  '10013999999'
         |--when categoryId  =  '201183501' then  '10022069999'
         |--when categoryId  =  '201169116' then  '10016059999'
         |--when categoryId  =  '201220510' then  '10022069999'
         |--when categoryId  =  '201248002' then  '10013029999'
         |--when categoryId  =  '201250101' then  '10013059999'
         |--when categoryId  =  '201229810' then  '10022069999'
         |--when categoryId  =  '201230514' then  '10011050499'
         |--when categoryId  =  '201236601' then  '10022050299'
         |--when categoryId  =  '201237616' then  '10022069999'
         |--when categoryId  =  '201228511' then  '10029999999'
         |--when categoryId  =  '201221102' then  '10022099999'
         |--when categoryId  =  '200926002' then  '10011999999'
         |--when categoryId  =  '201249901' then  '10013049999'
         |
         |
         |
         |else fourthCategoryId end as fourthCate
         |
         |from
         |data
      """.stripMargin).drop("firstCategoryId", "secondCategoryId", "thirdCategoryId", "fourthCategoryId")
      .withColumnRenamed("firstCate", "firstCategoryId")
      .withColumnRenamed("secondCate", "secondCategoryId")
      .withColumnRenamed("thirdCate", "thirdCategoryId")
      .withColumnRenamed("fourthCate", "fourthCategoryId")

    val valuess=
      datas.where(
          "categoryId  =  '201211201'  ")
    import org.apache.spark.sql.functions._
    valuess.selectExpr("categoryId","firstCategoryId","fourthCategoryId" ,"secondCategoryId","thirdCategoryId")
      .withColumn("products",lit("1000")).withColumn("last_categoryId",lit(col("categoryId"))).dropDuplicates("categoryId")
      .repartition(1).write.json("s3a://o2o-dimension-table/category_table/tm_cate_20191208_tmp")

   val value = datas.toJSON.rdd.map(line => {
     val lines = JSON.parseObject(line)
     val evaluates = lines.get("evaluates").toString
     var ev = "-1"
     var str = new JSONObject
     if (evaluates.equals("-1")) {
       ev = "{\"fuyi\":\"-1\"}"
     } else {
       str = JSON.parseObject(evaluates)
     }
     if (!ev.contains("fuyi")) {
       lines.put("evaluates", str)
     } else {
       val evs = JSON.parseObject(ev)
       lines.put("evaluates", evs)
     }
     lines
   })

   value.saveToEs(s"2020_${platform}/${platform}_2020_${month}", Map("es.mapping.id" -> "good_id"))


    sc.stop()
  }
}